package com.lucasx.spider;

import com.lucasx.entity.LishiWeather;
import com.lucasx.util.FileUtil;
import org.apache.http.Header;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicHeader;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * Created by Administrator on 2016/10/16.
 * <p>
 * 历史天气网爬虫 lishi.tianqi.com
 * <p>
 * Powered by <b>HttpClient4.5</b>
 */
public class LSTQSpider {

    private static List<LishiWeather> lishiWeatherList = new ArrayList<>();
    public static final String BROWSER_CHROME = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like" +
            " Gecko) Chrome/55.0.2882.4 Safari/537.36";

    /**
     * 从<b>lishi.tianqi.com</b>爬取数据(以天为单位),
     *
     * @param date   <b>FORMAT: yyyyMM</b>
     * @param pinYin
     */
    public void scrape(String pinYin, String date) throws IOException, InterruptedException {
        String url = "http://lishi.tianqi.com/" + pinYin + "/" + date + ".html";
        CloseableHttpClient httpClient = HttpClients.createDefault();
        HttpGet httpGet = new HttpGet(url);
        httpGet.setHeaders(new Header[]{new BasicHeader("Host", "lishi.tianqi.com"), new BasicHeader("Referer",
                "http://lishi.tianqi.com/" + pinYin + "/index.html"), new
                BasicHeader("User-Agent", LSTQSpider.BROWSER_CHROME), new BasicHeader("Accept", "text/html," +
                "application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"),
                new BasicHeader("Upgrade-Insecure-Requests", "1")});

        try {
            CloseableHttpResponse closeableHttpResponse = httpClient.execute(httpGet);
            if (closeableHttpResponse.getStatusLine().getStatusCode() == 200) {

//            logger.info("正在爬取时间为 " + date + " 的数据...");
                System.out.println("正在爬取时间为 " + date + " 的数据...");
                Document document = Jsoup.parse(EntityUtils.toString(closeableHttpResponse.getEntity()));
                Elements elements = document.select(".tqtongji2").first().getElementsByTag("ul");

                for (int i = 1; i < elements.size(); i++) {               // Here we've reached every <ul> tag...
                    Elements liElements = elements.get(i).getElementsByTag("li");
                    LishiWeather lishiWeather = new LishiWeather(liElements.get(0).text().toString().replace("-", "")
                            , Integer.parseInt(liElements.get(1).text().toString().trim()),
                            Integer.parseInt(liElements.get(2).text().toString().trim()), liElements.get(3).text()
                            .toString(), liElements.get(4).text().toString(),
                            liElements.get(5).text().toString());

                    lishiWeatherList.add(lishiWeather);
                }

            } else if (closeableHttpResponse.getStatusLine().getStatusCode() == 403) {
                System.err.println("您的访问受到限制!!");
            } else {
                System.err.println("Unknown error...");
            }
            closeableHttpResponse.close();
        } catch (NullPointerException nullPointerException) {
            System.err.println("日期 " + date + " 的数据不存在！");
        }

        Thread.sleep(3000);
    }

    public static void main(String[] args) {
        try {

            for (int i = 201501; i <= 201512; i++) {
                new LSTQSpider().scrape("jiaxing", i + "");
            }

            lishiWeatherList.forEach(lishiWeather -> {
                System.out.println(lishiWeather);
            });

            FileUtil.outCsv(lishiWeatherList, "D:/历史天气-嘉兴-2014.csv", false);
        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }

}